import requests
from lxml import etree
import js2xml
from DAO.DAO_major_catalog import DAO_major_catalog

class DATA_major_catalog:
    # 爬取所有学校的专业链接
    def major_catalog_link(self):
        urls = ['http://college.wendu.com/index.php?m=university&c=search&a=search&page={}'.format(str(i)) for i in range(1, 70)]
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"}
        for url in urls:
            print(url)
            response = requests.get(url, headers=headers)
            response_html = etree.HTML(response.content.decode())
            schools = response_html.xpath("//a[@class='school-list-itemLeft']")
            for school in schools:
                school_name = school.xpath(".//h3[@class='school-item-name']/text()")[0]
                major_link = "http://college.wendu.com/major_list-" + school.xpath("./@href")[0].split("-")[1]+"-1-a-a"
                print(school_name, major_link)
                self.major_catalog(major_link)
                #break
            #break

    def major_catalog(self, url):
        # url = 'http://college.wendu.com/major_list-1-1-a-a'
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"}
        response = requests.get(url, headers=headers)
        response_str = response.content.decode()
        response_html = etree.HTML(response_str)
        script_list = response_html.xpath("//script/text()")
        #print(script_list[6])
        try:
            script_xml = js2xml.parse(script_list[11], encoding='utf-8', debug=False)
            # print(script_xml)
            script_tree = js2xml.pretty_print(script_xml)
            # print(script_tree)
            script_data_html = etree.HTML(script_tree)
            div_all = script_data_html.xpath('//var[@name="datas"]/array/object')
            for div_one in div_all:
                maj_code = div_one.xpath('.//property[@name="major_code"]/string/text()')[0]
                maj_name = div_one.xpath('.//property[@name="major_name"]/string/text()')[0]
                maj_college_name = div_one.xpath('.//property[@name="academy_name"]/string/text()')[0]
                maj_uni_name = response_html.xpath('//p[@class="where"]/a[2]/text()')[0]
                dao = DAO_major_catalog()
                print(maj_code, maj_name, maj_college_name, maj_uni_name)
                dao.add(maj_code, maj_name, maj_college_name, maj_uni_name)
        except:
            print("有错误》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》》")


if __name__ == '__main__':
    print("1、爬取所有院校招生简章数据")
    print("2、爬取一个院校招生简章数据")
    print("3、清空数据库")
    print("4、输出所有数据")
    res = int(input())
    if res == 1:
        major = DATA_major_catalog()
        major.major_catalog_link()
    elif res == 2:
        print("暂无")
    elif res == 3:
        dao = DAO_major_catalog()
        dao.truncate()
    elif res == 4:
        dao = DAO_major_catalog()
        dao.sel_all()






# print(maj_college_name_list)
